{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adapted from Amy Steicker's tutorial for ICESat-2 data download using bounding box around Grand Mesa, CO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import requests\n",
    "import getpass\n",
    "import socket\n",
    "import json\n",
    "import zipfile\n",
    "import io\n",
    "import math\n",
    "import os\n",
    "import shutil\n",
    "import pprint\n",
    "import time\n",
    "import geopandas as gpd\n",
    "import matplotlib.pyplot as plt\n",
    "import fiona\n",
    "import h5py\n",
    "import re\n",
    "\n",
    "# To read KML files with geopandas, we will need to enable KML support in fiona (disabled by default)\n",
    "fiona.drvsupport.supported_drivers['LIBKML'] = 'rw'\n",
    "# from shapely.geometry import Polygon, mapping\n",
    "# from shapely.geometry.polygon import orient\n",
    "from statistics import mean\n",
    "from requests.auth import HTTPBasicAuth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Earthdata Login password:  ················\n"
     ]
    }
   ],
   "source": [
    "# Create token\n",
    "\n",
    "# Earthdata Login credentials\n",
    "\n",
    "# Enter your Earthdata Login user name\n",
    "uid = 'jmichellehu'\n",
    "# Enter your email address associated with your Earthdata Login account\n",
    "email = 'jmhu@uw.edu'\n",
    "pswd = getpass.getpass('Earthdata Login password: ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A424425C-D4B2-71F8-8AF0-874A25BD3177\n"
     ]
    }
   ],
   "source": [
    "# Request token from Common Metadata Repository using Earthdata credentials\n",
    "token_api_url = 'https://cmr.earthdata.nasa.gov/legacy-services/rest/tokens'\n",
    "hostname = socket.gethostname()\n",
    "ip = socket.gethostbyname(hostname)\n",
    "\n",
    "data = {\n",
    "    'token': {\n",
    "        'username': uid,\n",
    "        'password': pswd,\n",
    "        'client_id': 'NSIDC_client_id',\n",
    "        'user_ip_address': ip\n",
    "    }\n",
    "}\n",
    "headers={'Accept': 'application/json'}\n",
    "response = requests.post(token_api_url, json=data, headers=headers)\n",
    "token = json.loads(response.content)['token']['id']\n",
    "print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input data set ID (e.g. ATL06) of interest here, also known as \"short name\".\n",
    "\n",
    "short_name = 'ATL06'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get json response from CMR collection metadata and print results. This provides high-level metadata on a data set or \"collection\", provide in json format.\n",
    "\n",
    "params = {\n",
    "    'short_name': short_name\n",
    "}\n",
    "\n",
    "cmr_collections_url = 'https://cmr.earthdata.nasa.gov/search/collections.json'\n",
    "response = requests.get(cmr_collections_url, params=params)\n",
    "results = json.loads(response.content)\n",
    "# pprint.pprint(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "001\n"
     ]
    }
   ],
   "source": [
    "# Find all instances of 'version_id' in metadata and print most recent version number\n",
    "\n",
    "versions = [i['version_id'] for i in results['feed']['entry']]\n",
    "latest_version = max(versions)\n",
    "print(latest_version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-10-16T00:00:00Z,2019-06-17T23:59:59Z\n"
     ]
    }
   ],
   "source": [
    "# Input temporal range if you have a specific range, otherwise, this is unnecessary\n",
    "\n",
    "# Input start date in yyyy-MM-dd format\n",
    "start_date = '2018-10-16'\n",
    "# Input start time in HH:mm:ss format\n",
    "start_time = '00:00:00'\n",
    "# Input end date in yyyy-MM-dd format\n",
    "end_date = '2019-06-17'\n",
    "# Input end time in HH:mm:ss format\n",
    "end_time = '23:59:59'\n",
    "\n",
    "temporal = start_date + 'T' + start_time + 'Z' + ',' + end_date + 'T' + end_time + 'Z'\n",
    "print(temporal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-108.3,38.8,-107.5,39.3\n"
     ]
    }
   ],
   "source": [
    "# Commenting for tutorial since we will be walking through option 3 (spatial file input) together\n",
    "# Bounding Box spatial parameter in 'W,S,E,N' format\n",
    "\n",
    "# Input bounding box\n",
    "# Input lower left longitude in decimal degrees\n",
    "LL_lon = '-108.3'\n",
    "# Input lower left latitude in decimal degrees\n",
    "LL_lat = '38.8'\n",
    "# Input upper right longitude in decimal degrees\n",
    "UR_lon = '-107.5'\n",
    "# Input upper right latitude in decimal degrees\n",
    "UR_lat = '39.3'\n",
    "\n",
    "bounding_box = LL_lon + ',' + LL_lat + ',' + UR_lon + ',' + UR_lat\n",
    "# aoi value used for CMR params below\n",
    "aoi = '1'\n",
    "print(bounding_box)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CMR search parameters:  {'short_name': 'ATL06', 'version': '001', 'temporal': '2018-10-16T00:00:00Z,2019-06-17T23:59:59Z', 'page_size': 100, 'page_num': 1, 'bounding_box': '-108.3,38.8,-107.5,39.3'}\n"
     ]
    }
   ],
   "source": [
    "#Create CMR parameters used for granule search. Modify params depending on bounding_box or polygon input.\n",
    "\n",
    "if aoi == '1':\n",
    "# bounding box input:\n",
    "    params = {\n",
    "    'short_name': short_name,\n",
    "    'version': latest_version,\n",
    "    'temporal': temporal,\n",
    "    'page_size': 100,\n",
    "    'page_num': 1,\n",
    "    'bounding_box': bounding_box\n",
    "    }\n",
    "else:\n",
    "    \n",
    "# If polygon input (either via coordinate pairs or shapefile/KML/KMZ):\n",
    "    params = {\n",
    "    'short_name': short_name,\n",
    "    'version': latest_version,\n",
    "    'temporal': temporal,\n",
    "    'page_size': 100,\n",
    "    'page_num': 1,\n",
    "    'polygon': polygon,\n",
    "    }\n",
    "\n",
    "print('CMR search parameters: ', params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Query number of granules using our (paging over results)\n",
    "\n",
    "granule_search_url = 'https://cmr.earthdata.nasa.gov/search/granules'\n",
    "\n",
    "granules = []\n",
    "while True:\n",
    "    response = requests.get(granule_search_url, params=params, headers=headers)\n",
    "    results = json.loads(response.content)\n",
    "\n",
    "    if len(results['feed']['entry']) == 0:\n",
    "        # Out of results, so break out of loop\n",
    "        break\n",
    "\n",
    "    # Collect results and increment page_num\n",
    "    granules.extend(results['feed']['entry'])\n",
    "    params['page_num'] += 1\n",
    "\n",
    "    \n",
    "# Get number of granules over my area and time of interest\n",
    "len(granules)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21.039645126892857\n",
      "294.55503177649996\n"
     ]
    }
   ],
   "source": [
    "granule_sizes = [float(granule['granule_size']) for granule in granules]\n",
    "\n",
    "# Average size of granules in MB\n",
    "print(mean(granule_sizes))\n",
    "\n",
    "# Total size in MB\n",
    "print(sum(granule_sizes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://n5eil02u.ecs.nsidc.org/egi/capabilities/ATL06.001.xml\n"
     ]
    }
   ],
   "source": [
    "# Query service capability URL \n",
    "\n",
    "from xml.etree import ElementTree as ET\n",
    "\n",
    "capability_url = f'https://n5eil02u.ecs.nsidc.org/egi/capabilities/{short_name}.{latest_version}.xml'\n",
    "\n",
    "print(capability_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create session to store cookie and pass credentials to capabilities url\n",
    "\n",
    "session = requests.session()\n",
    "s = session.get(capability_url)\n",
    "response = session.get(s.url,auth=(uid,pswd))\n",
    "\n",
    "root = ET.fromstring(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# collect lists with each service option\n",
    "\n",
    "subagent = [subset_agent.attrib for subset_agent in root.iter('SubsetAgent')]\n",
    "\n",
    "# variable subsetting\n",
    "variables = [SubsetVariable.attrib for SubsetVariable in root.iter('SubsetVariable')]  \n",
    "variables_raw = [variables[i]['value'] for i in range(len(variables))]\n",
    "variables_join = [''.join(('/',v)) if v.startswith('/') == False else v for v in variables_raw] \n",
    "variable_vals = [v.replace(':', '/') for v in variables_join]\n",
    "\n",
    "# reformatting\n",
    "formats = [Format.attrib for Format in root.iter('Format')]\n",
    "format_vals = [formats[i]['value'] for i in range(len(formats))]\n",
    "format_vals.remove('')\n",
    "\n",
    "# reprojection only applicable on ICESat-2 L3B products, yet to be available. \n",
    "\n",
    "# reformatting options that support reprojection\n",
    "normalproj = [Projections.attrib for Projections in root.iter('Projections')]\n",
    "normalproj_vals = []\n",
    "normalproj_vals.append(normalproj[0]['normalProj'])\n",
    "format_proj = normalproj_vals[0].split(',')\n",
    "format_proj.remove('')\n",
    "format_proj.append('No reformatting')\n",
    "\n",
    "#reprojection options\n",
    "projections = [Projection.attrib for Projection in root.iter('Projection')]\n",
    "proj_vals = []\n",
    "for i in range(len(projections)):\n",
    "    if (projections[i]['value']) != 'NO_CHANGE' :\n",
    "        proj_vals.append(projections[i]['value'])\n",
    "        \n",
    "# reformatting options that do not support reprojection\n",
    "no_proj = [i for i in format_vals if i not in format_proj]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'id': 'ICESAT2', 'spatialSubsetting': 'true', 'spatialSubsettingShapefile': 'true', 'temporalSubsetting': 'true', 'type': 'both', 'maxGransSyncRequest': '100', 'maxGransAsyncRequest': '2000'}]\n"
     ]
    }
   ],
   "source": [
    "print(subagent)\n",
    "if len(subagent) < 1 :\n",
    "    agent = 'NO'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bounding box subsetting (bbox) in same format as bounding_box\n",
    "bbox = bounding_box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-10-16T00:00:00,2019-06-17T23:59:59\n"
     ]
    }
   ],
   "source": [
    "# Temporal subsetting KVP\n",
    "\n",
    "timevar = start_date + 'T' + start_time + ',' + end_date + 'T' + end_time\n",
    "print(timevar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "627"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(variable_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pprint.pprint(variable_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify variables of interest\n",
    "\n",
    "coverage = '/ancillary_data/atlas_sdp_gps_epoch,\\\n",
    "/gt1l/land_ice_segments/atl06_quality_summary,\\\n",
    "/gt1l/land_ice_segments/delta_time,\\\n",
    "/gt1l/land_ice_segments/h_li,\\\n",
    "/gt1l/land_ice_segments/h_li_sigma,\\\n",
    "/gt1l/land_ice_segments/latitude,\\\n",
    "/gt1l/land_ice_segments/longitude,\\\n",
    "/gt1l/land_ice_segments/segment_id,\\\n",
    "/gt1l/land_ice_segments/sigma_geo_h,\\\n",
    "/gt1r/land_ice_segments/atl06_quality_summary,\\\n",
    "/gt1r/land_ice_segments/delta_time,\\\n",
    "/gt1r/land_ice_segments/h_li,\\\n",
    "/gt1r/land_ice_segments/h_li_sigma,\\\n",
    "/gt1r/land_ice_segments/latitude,\\\n",
    "/gt1r/land_ice_segments/longitude,\\\n",
    "/gt1r/land_ice_segments/segment_id,\\\n",
    "/gt1r/land_ice_segments/sigma_geo_h,\\\n",
    "/gt2l/land_ice_segments/atl06_quality_summary,\\\n",
    "/gt2l/land_ice_segments/delta_time,\\\n",
    "/gt2l/land_ice_segments/h_li,\\\n",
    "/gt2l/land_ice_segments/h_li_sigma,\\\n",
    "/gt2l/land_ice_segments/latitude,\\\n",
    "/gt2l/land_ice_segments/longitude,\\\n",
    "/gt2l/land_ice_segments/segment_id,\\\n",
    "/gt2l/land_ice_segments/sigma_geo_h,\\\n",
    "/gt2r/land_ice_segments/atl06_quality_summary,\\\n",
    "/gt2r/land_ice_segments/delta_time,\\\n",
    "/gt2r/land_ice_segments/h_li,\\\n",
    "/gt2r/land_ice_segments/h_li_sigma,\\\n",
    "/gt2r/land_ice_segments/latitude,\\\n",
    "/gt2r/land_ice_segments/longitude,\\\n",
    "/gt2r/land_ice_segments/segment_id,\\\n",
    "/gt2r/land_ice_segments/sigma_geo_h,\\\n",
    "/gt3l/land_ice_segments/atl06_quality_summary,\\\n",
    "/gt3l/land_ice_segments/delta_time,\\\n",
    "/gt3l/land_ice_segments/h_li,\\\n",
    "/gt3l/land_ice_segments/h_li_sigma,\\\n",
    "/gt3l/land_ice_segments/latitude,\\\n",
    "/gt3l/land_ice_segments/longitude,\\\n",
    "/gt3l/land_ice_segments/segment_id,\\\n",
    "/gt3l/land_ice_segments/sigma_geo_h,\\\n",
    "/gt3r/land_ice_segments/atl06_quality_summary,\\\n",
    "/gt3r/land_ice_segments/delta_time,\\\n",
    "/gt3r/land_ice_segments/h_li,\\\n",
    "/gt3r/land_ice_segments/h_li_sigma,\\\n",
    "/gt3r/land_ice_segments/latitude,\\\n",
    "/gt3r/land_ice_segments/longitude,\\\n",
    "/gt3r/land_ice_segments/segment_id,\\\n",
    "/gt3r/land_ice_segments/sigma_geo_h,\\\n",
    "/orbit_info/cycle_number,\\\n",
    "/orbit_info/rgt,\\\n",
    "/orbit_info/orbit_number' \n",
    "\n",
    "# # Other variables to add to coverage \n",
    "# /gt1l/land_ice_segments/geophysical/r_eff,\\\n",
    "# /gt1l/land_ice_segments/ground_track/x_atc,\\\n",
    "# /gt1l/land_ice_segments/n_fit_photons,\\\n",
    "# /gt1l/land_ice_segments/w_surface_window_final,\\\n",
    "# /gt1l/land_ice_segments/h_rms_misft,\\\n",
    "# /gt1l/land_ice_segments/h_robust_sprd,\\\n",
    "# /gt1l/land_ice_segments/snr,\\\n",
    "# /gt1l/land_ice_segments/snr_significance,\\\n",
    "# /gt1l/land_ice_segments/dh_fit_dx,\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set NSIDC data access base URL\n",
    "base_url = 'https://n5eil02u.ecs.nsidc.org/egi/request'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "# Set number of granules requested per order, which we will initially set to 10.\n",
    "# changed to 20 as Grand Mesa has 14 granules over this time period\n",
    "page_size = 20\n",
    "\n",
    "#Determine number of pages basd on page_size and total granules. Loop requests by this value\n",
    "page_num = math.ceil(len(granules)/page_size)\n",
    "\n",
    "#Set request mode. \n",
    "request_mode = 'async'\n",
    "\n",
    "# Determine how many individual orders we will request based on the number of granules requested\n",
    "\n",
    "print(page_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'short_name': 'ATL06', 'version': '001', 'temporal': '2018-10-16T00:00:00Z,2019-06-17T23:59:59Z', 'time': '2018-10-16T00:00:00,2019-06-17T23:59:59', 'bounding_box': '-108.3,38.8,-107.5,39.3', 'bbox': '-108.3,38.8,-107.5,39.3', 'Coverage': '/ancillary_data/atlas_sdp_gps_epoch,/gt1l/land_ice_segments/atl06_quality_summary,/gt1l/land_ice_segments/delta_time,/gt1l/land_ice_segments/h_li,/gt1l/land_ice_segments/h_li_sigma,/gt1l/land_ice_segments/latitude,/gt1l/land_ice_segments/longitude,/gt1l/land_ice_segments/segment_id,/gt1l/land_ice_segments/sigma_geo_h,/gt1r/land_ice_segments/atl06_quality_summary,/gt1r/land_ice_segments/delta_time,/gt1r/land_ice_segments/h_li,/gt1r/land_ice_segments/h_li_sigma,/gt1r/land_ice_segments/latitude,/gt1r/land_ice_segments/longitude,/gt1r/land_ice_segments/segment_id,/gt1r/land_ice_segments/sigma_geo_h,/gt2l/land_ice_segments/atl06_quality_summary,/gt2l/land_ice_segments/delta_time,/gt2l/land_ice_segments/h_li,/gt2l/land_ice_segments/h_li_sigma,/gt2l/land_ice_segments/latitude,/gt2l/land_ice_segments/longitude,/gt2l/land_ice_segments/segment_id,/gt2l/land_ice_segments/sigma_geo_h,/gt2r/land_ice_segments/atl06_quality_summary,/gt2r/land_ice_segments/delta_time,/gt2r/land_ice_segments/h_li,/gt2r/land_ice_segments/h_li_sigma,/gt2r/land_ice_segments/latitude,/gt2r/land_ice_segments/longitude,/gt2r/land_ice_segments/segment_id,/gt2r/land_ice_segments/sigma_geo_h,/gt3l/land_ice_segments/atl06_quality_summary,/gt3l/land_ice_segments/delta_time,/gt3l/land_ice_segments/h_li,/gt3l/land_ice_segments/h_li_sigma,/gt3l/land_ice_segments/latitude,/gt3l/land_ice_segments/longitude,/gt3l/land_ice_segments/segment_id,/gt3l/land_ice_segments/sigma_geo_h,/gt3r/land_ice_segments/atl06_quality_summary,/gt3r/land_ice_segments/delta_time,/gt3r/land_ice_segments/h_li,/gt3r/land_ice_segments/h_li_sigma,/gt3r/land_ice_segments/latitude,/gt3r/land_ice_segments/longitude,/gt3r/land_ice_segments/segment_id,/gt3r/land_ice_segments/sigma_geo_h,/orbit_info/cycle_number,/orbit_info/rgt,/orbit_info/orbit_number', 'request_mode': 'async', 'page_size': 20, 'token': 'A424425C-D4B2-71F8-8AF0-874A25BD3177', 'email': 'jmhu@uw.edu'}\n"
     ]
    }
   ],
   "source": [
    "subset_params = {\n",
    "    'short_name': short_name, \n",
    "    'version': latest_version, \n",
    "    'temporal': temporal, \n",
    "    'time': timevar, \n",
    "    'bounding_box': bounding_box,\n",
    "    'bbox': bbox,\n",
    "    'Coverage': coverage, \n",
    "    'request_mode': request_mode, \n",
    "    'page_size': page_size,  \n",
    "    'token': token, \n",
    "    'email': email, \n",
    "    }\n",
    "print(subset_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define paths for output folders\n",
    "sopath = '/home/jovyan/data/s_outputs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Make subsetted outputs directory\n",
    "# path = sopath\n",
    "# if not os.path.exists(path):\n",
    "#     os.mkdir(path)\n",
    "# path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = str(os.getcwd() + '/Outputs')\n",
    "if not os.path.exists(path):\n",
    "    os.mkdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Order:  1\n",
      "Request HTTP response:  201\n",
      "Order request URL:  https://n5eil02u.ecs.nsidc.org/egi/request?short_name=ATL06&version=001&temporal=2018-10-16T00%3A00%3A00Z%2C2019-06-17T23%3A59%3A59Z&time=2018-10-16T00%3A00%3A00%2C2019-06-17T23%3A59%3A59&bounding_box=-108.3%2C38.8%2C-107.5%2C39.3&bbox=-108.3%2C38.8%2C-107.5%2C39.3&Coverage=%2Fancillary_data%2Fatlas_sdp_gps_epoch%2C%2Fgt1l%2Fland_ice_segments%2Fatl06_quality_summary%2C%2Fgt1l%2Fland_ice_segments%2Fdelta_time%2C%2Fgt1l%2Fland_ice_segments%2Fh_li%2C%2Fgt1l%2Fland_ice_segments%2Fh_li_sigma%2C%2Fgt1l%2Fland_ice_segments%2Flatitude%2C%2Fgt1l%2Fland_ice_segments%2Flongitude%2C%2Fgt1l%2Fland_ice_segments%2Fsegment_id%2C%2Fgt1l%2Fland_ice_segments%2Fsigma_geo_h%2C%2Fgt1r%2Fland_ice_segments%2Fatl06_quality_summary%2C%2Fgt1r%2Fland_ice_segments%2Fdelta_time%2C%2Fgt1r%2Fland_ice_segments%2Fh_li%2C%2Fgt1r%2Fland_ice_segments%2Fh_li_sigma%2C%2Fgt1r%2Fland_ice_segments%2Flatitude%2C%2Fgt1r%2Fland_ice_segments%2Flongitude%2C%2Fgt1r%2Fland_ice_segments%2Fsegment_id%2C%2Fgt1r%2Fland_ice_segments%2Fsigma_geo_h%2C%2Fgt2l%2Fland_ice_segments%2Fatl06_quality_summary%2C%2Fgt2l%2Fland_ice_segments%2Fdelta_time%2C%2Fgt2l%2Fland_ice_segments%2Fh_li%2C%2Fgt2l%2Fland_ice_segments%2Fh_li_sigma%2C%2Fgt2l%2Fland_ice_segments%2Flatitude%2C%2Fgt2l%2Fland_ice_segments%2Flongitude%2C%2Fgt2l%2Fland_ice_segments%2Fsegment_id%2C%2Fgt2l%2Fland_ice_segments%2Fsigma_geo_h%2C%2Fgt2r%2Fland_ice_segments%2Fatl06_quality_summary%2C%2Fgt2r%2Fland_ice_segments%2Fdelta_time%2C%2Fgt2r%2Fland_ice_segments%2Fh_li%2C%2Fgt2r%2Fland_ice_segments%2Fh_li_sigma%2C%2Fgt2r%2Fland_ice_segments%2Flatitude%2C%2Fgt2r%2Fland_ice_segments%2Flongitude%2C%2Fgt2r%2Fland_ice_segments%2Fsegment_id%2C%2Fgt2r%2Fland_ice_segments%2Fsigma_geo_h%2C%2Fgt3l%2Fland_ice_segments%2Fatl06_quality_summary%2C%2Fgt3l%2Fland_ice_segments%2Fdelta_time%2C%2Fgt3l%2Fland_ice_segments%2Fh_li%2C%2Fgt3l%2Fland_ice_segments%2Fh_li_sigma%2C%2Fgt3l%2Fland_ice_segments%2Flatitude%2C%2Fgt3l%2Fland_ice_segments%2Flongitude%2C%2Fgt3l%2Fland_ice_segments%2Fsegment_id%2C%2Fgt3l%2Fland_ice_segments%2Fsigma_geo_h%2C%2Fgt3r%2Fland_ice_segments%2Fatl06_quality_summary%2C%2Fgt3r%2Fland_ice_segments%2Fdelta_time%2C%2Fgt3r%2Fland_ice_segments%2Fh_li%2C%2Fgt3r%2Fland_ice_segments%2Fh_li_sigma%2C%2Fgt3r%2Fland_ice_segments%2Flatitude%2C%2Fgt3r%2Fland_ice_segments%2Flongitude%2C%2Fgt3r%2Fland_ice_segments%2Fsegment_id%2C%2Fgt3r%2Fland_ice_segments%2Fsigma_geo_h%2C%2Forbit_info%2Fcycle_number%2C%2Forbit_info%2Frgt%2C%2Forbit_info%2Forbit_number&request_mode=async&page_size=20&token=A424425C-D4B2-71F8-8AF0-874A25BD3177&email=jmhu%40uw.edu&page_num=1\n",
      "Order request response XML content:  b'<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\\n<eesi:agentResponse xsi:schemaLocation=\"http://eosdis.nasa.gov/esi/rsp/e https://newsroom.gsfc.nasa.gov/esi/8.1/schemas/ESIAgentResponseExternal.xsd\" xmlns=\"\" xmlns:iesi=\"http://eosdis.nasa.gov/esi/rsp/i\" xmlns:ssw=\"http://newsroom.gsfc.nasa.gov/esi/rsp/ssw\" xmlns:eesi=\"http://eosdis.nasa.gov/esi/rsp/e\" xmlns:esi=\"http://eosdis.nasa.gov/esi/rsp\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\\n    <order>\\n        <orderId>5000000319526</orderId>\\n        <Instructions>You may receive an email about your order if you specified an EMAIL address. &lt;br/&gt;&lt;br/&gt;The instructions used to process this order are:  Bounding Box(es)=-108.3,38.8,-107.5,39.3. Band(s)=/ancillary_data/atlas_sdp_gps_epoch,/gt1l/land_ice_segments/atl06_quality_summary,/gt1l/land_ice_segments/delta_time,/gt1l/land_ice_segments/h_li,/gt1l/land_ice_segments/h_li_sigma,/gt1l/land_ice_segments/latitude,/gt1l/land_ice_segments/longitude,/gt1l/land_ice_segments/segment_id,/gt1l/land_ice_segments/sigma_geo_h,/gt1r/land_ice_segments/atl06_quality_summary,/gt1r/land_ice_segments/delta_time,/gt1r/land_ice_segments/h_li,/gt1r/land_ice_segments/h_li_sigma,/gt1r/land_ice_segments/latitude,/gt1r/land_ice_segments/longitude,/gt1r/land_ice_segments/segment_id,/gt1r/land_ice_segments/sigma_geo_h,/gt2l/land_ice_segments/atl06_quality_summary,/gt2l/land_ice_segments/delta_time,/gt2l/land_ice_segments/h_li,/gt2l/land_ice_segments/h_li_sigma,/gt2l/land_ice_segments/latitude,/gt2l/land_ice_segments/longitude,/gt2l/land_ice_segments/segment_id,/gt2l/land_ice_segments/sigma_geo_h,/gt2r/land_ice_segments/atl06_quality_summary,/gt2r/land_ice_segments/delta_time,/gt2r/land_ice_segments/h_li,/gt2r/land_ice_segments/h_li_sigma,/gt2r/land_ice_segments/latitude,/gt2r/land_ice_segments/longitude,/gt2r/land_ice_segments/segment_id,/gt2r/land_ice_segments/sigma_geo_h,/gt3l/land_ice_segments/atl06_quality_summary,/gt3l/land_ice_segments/delta_time,/gt3l/land_ice_segments/h_li,/gt3l/land_ice_segments/h_li_sigma,/gt3l/land_ice_segments/latitude,/gt3l/land_ice_segments/longitude,/gt3l/land_ice_segments/segment_id,/gt3l/land_ice_segments/sigma_geo_h,/gt3r/land_ice_segments/atl06_quality_summary,/gt3r/land_ice_segments/delta_time,/gt3r/land_ice_segments/h_li,/gt3r/land_ice_segments/h_li_sigma,/gt3r/land_ice_segments/latitude,/gt3r/land_ice_segments/longitude,/gt3r/land_ice_segments/segment_id,/gt3r/land_ice_segments/sigma_geo_h,/orbit_info/cycle_number,/orbit_info/rgt,/orbit_info/orbit_number. Granule id(s)=SC:ATL06.001:161028994,SC:ATL06.001:161138682,SC:ATL06.001:161152465,SC:ATL06.001:161084250,SC:ATL06.001:161159414,SC:ATL06.001:161209871,SC:ATL06.001:161210043,SC:ATL06.001:161238717,SC:ATL06.001:161252239,SC:ATL06.001:161255398,SC:ATL06.001:161313271,SC:ATL06.001:161334410,SC:ATL06.001:161322380,SC:ATL06.001:161336208. Temporal search start=2018-10-16T00:00:00 end=2019-06-17T23:59:59. Email address=jmhu@uw.edu. Processing tool=ICESAT2.</Instructions>\\n    </order>\\n    <contactInformation>\\n        <contactName>NSIDC User Services</contactName>\\n        <contactEmail>nsidc@nsidc.org</contactEmail>\\n    </contactInformation>\\n    <processInfo>\\n        <processDuration>PT0.129S</processDuration>\\n        <subagentId>ICESAT2</subagentId>\\n    </processInfo>\\n    <requestStatus>\\n        <status>processing</status>\\n        <numberProcessed>0</numberProcessed>\\n        <totalNumber>14</totalNumber>\\n    </requestStatus>\\n</eesi:agentResponse>\\n'\n",
      "order ID:  5000000319526\n",
      "status URL:  https://n5eil02u.ecs.nsidc.org/egi/request/5000000319526\n",
      "HTTP response from order response URL:  201\n",
      "Data request  1  is submitting...\n",
      "Initial request status is  processing\n",
      "Status is not complete. Trying again.\n",
      "Retry request status is:  complete_with_errors\n",
      "error messages:\n",
      "['161028994:NoMatchingData - No data found that matched subset constraints. '\n",
      " 'Exit code 3.',\n",
      " '161152465:NoMatchingData - No data found that matched subset constraints. '\n",
      " 'Exit code 3.',\n",
      " '161084250:NoMatchingData - No data found that matched subset constraints. '\n",
      " 'Exit code 3.',\n",
      " '161255398:NoMatchingData - No data found that matched subset constraints. '\n",
      " 'Exit code 3.',\n",
      " '161334410:NoMatchingData - No data found that matched subset constraints. '\n",
      " 'Exit code 3.',\n",
      " '161322380:NoMatchingData - No data found that matched subset constraints. '\n",
      " 'Exit code 3.',\n",
      " 'PT2.940S',\n",
      " 'ICESAT2']\n",
      "Zip download URL:  https://n5eil02u.ecs.nsidc.org/esir/5000000319526.zip\n",
      "Beginning download of zipped output...\n",
      "Data request 1 is complete.\n"
     ]
    }
   ],
   "source": [
    "# Request data service for each page number, and unzip outputs\n",
    "\n",
    "for i in range(page_num):\n",
    "    page_val = i + 1\n",
    "    print('Order: ', page_val)\n",
    "    subset_params.update( {'page_num': page_val} )\n",
    "    \n",
    "## Post polygon to API endpoint for polygon subsetting to subset based on original, non-simplified KML file\n",
    "\n",
    "#     shape_post = {'shapefile': open(kml_filepath, 'rb')}\n",
    "#     request = session.post(base_url, params=subset_params, files=shape_post) \n",
    "    \n",
    "# FOR ALL OTHER REQUESTS THAT DO NOT UTILIZED AN UPLOADED POLYGON FILE, USE A GET REQUEST INSTEAD OF POST:\n",
    "    request = session.get(base_url, params=subset_params)\n",
    "    \n",
    "    print('Request HTTP response: ', request.status_code)\n",
    "\n",
    "# Raise bad request: Loop will stop for bad response code.\n",
    "    request.raise_for_status()\n",
    "    print('Order request URL: ', request.url)\n",
    "    esir_root = ET.fromstring(request.content)\n",
    "    print('Order request response XML content: ', request.content)\n",
    "\n",
    "# Look up order ID\n",
    "    orderlist = []   \n",
    "    for order in esir_root.findall(\"./order/\"):\n",
    "        orderlist.append(order.text)\n",
    "    orderID = orderlist[0]\n",
    "    print('order ID: ', orderID)\n",
    "\n",
    "# Create status URL\n",
    "    statusURL = base_url + '/' + orderID\n",
    "    print('status URL: ', statusURL)\n",
    "\n",
    "# Find order status\n",
    "    request_response = session.get(statusURL)    \n",
    "    print('HTTP response from order response URL: ', request_response.status_code)\n",
    "    \n",
    "# Raise bad request: Loop will stop for bad response code.\n",
    "    request_response.raise_for_status()\n",
    "    request_root = ET.fromstring(request_response.content)\n",
    "    statuslist = []\n",
    "    for status in request_root.findall(\"./requestStatus/\"):\n",
    "        statuslist.append(status.text)\n",
    "    status = statuslist[0]\n",
    "    print('Data request ', page_val, ' is submitting...')\n",
    "    print('Initial request status is ', status)\n",
    "\n",
    "# Continue to loop while request is still processing\n",
    "    while status == 'pending' or status == 'processing': \n",
    "        print('Status is not complete. Trying again.')\n",
    "        time.sleep(10)\n",
    "        loop_response = session.get(statusURL)\n",
    "\n",
    "# Raise bad request: Loop will stop for bad response code.\n",
    "        loop_response.raise_for_status()\n",
    "        loop_root = ET.fromstring(loop_response.content)\n",
    "\n",
    "# Find status\n",
    "        statuslist = []\n",
    "        for status in loop_root.findall(\"./requestStatus/\"):\n",
    "            statuslist.append(status.text)\n",
    "        status = statuslist[0]\n",
    "        print('Retry request status is: ', status)\n",
    "        if status == 'pending' or status == 'processing':\n",
    "            continue\n",
    "\n",
    "# Order can either complete, complete_with_errors, or fail:\n",
    "# Provide complete_with_errors error message:\n",
    "    if status == 'complete_with_errors' or status == 'failed':\n",
    "        messagelist = []\n",
    "        for message in loop_root.findall(\"./processInfo/\"):\n",
    "            messagelist.append(message.text)\n",
    "        print('error messages:')\n",
    "        pprint.pprint(messagelist)\n",
    "\n",
    "# Download zipped order if status is complete or complete_with_errors\n",
    "    if status == 'complete' or status == 'complete_with_errors':\n",
    "        downloadURL = 'https://n5eil02u.ecs.nsidc.org/esir/' + orderID + '.zip'\n",
    "        print('Zip download URL: ', downloadURL)\n",
    "        print('Beginning download of zipped output...')\n",
    "        zip_response = session.get(downloadURL)\n",
    "        # Raise bad request: Loop will stop for bad response code.\n",
    "        zip_response.raise_for_status()\n",
    "        with zipfile.ZipFile(io.BytesIO(zip_response.content)) as z:\n",
    "            z.extractall(path)\n",
    "        print('Data request', page_val, 'is complete.')\n",
    "    else: print('Request failed.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
